NUMERIC = '0123456789'
HEX = NUMERIC+'abcdefABCDEF'
alpha = 'abcdefghijklmnopqrstuvwxyz'
ALPHA = alpha + alpha.upper()
ALPHANUMERIC = ALPHA + NUMERIC
WHITESPACE = ' \t\n\r'
ENDSTATEMENT = ';\n'
operators = [
    '!',
    '~',
    '+',
    '-',
    '*',
    '^',
    '/',
    '=',
    '%',
    '>',
    '<',
    '&',
    '|',
    '==',
    '++',
    '--',
    '>=',
    '<=',
    '&&',
    '||',
    '>>',
    '<<',
    '>>>',
    '<<<'
]
OPERATORS = ''
longest_operator = 0
for operator in operators: 
    if len(operator) > longest_operator: longest_operator = len(operator)
    for char in operator:
        if not char in OPERATORS: OPERATORS += char

class InputStream(object):
    End = '\0'
    def __init__(self):
        self.__index = 0
        self.__length = 0
        self.__source = ''
    def __init__(self, source):
        self.__index = 0
        self.__length = len(source)
        self.__source = source        
    def input(self, source):
        self.__source = source
        self.__length = len(source)
    def source(self):
        return self.__source
    def next(self, n=1):
        ret = self.peek(0, n)
        self.__index += n
        if self.__index  > self.__length: self.__index == self.__length
        return ret
    def peek(self, index=0, n=1):
        if self.__index + index >= self.__length: return InputStream.End
        if self.__index + index + n > self.__length:
            n = self.__index + index + n - self.__length
            return self.__source[self.__index+index:self.__index+index+n] + InputStream.End
        else:
            return self.__source[self.__index+index:self.__index+index+n]
    def index(self):
        return self.__index

class Lexer(object): pass
class LexerException(Exception): pass

class OperatorLexer(Lexer):
    class Exception(LexerException): pass
    class Operator(object):
        def __init__(self, operator):
            self.operator = operator
        def __repr__(self):
            return self.operator
    def lex(self, input):
        if not isinstance(input, InputStream): input = InputStream(str(input))
        if not input.peek() in OPERATORS: return None
        state = OperatorLexer.Operator('')
        if input.peek(0, 3) in operators:
            state.operator = input.next(3)
            return state
        if input.peek(0, 2) in operators:            
            state.operator = input.next(2)
            return state
        state.operator = input.next()
        return state

class WhitespaceLexer(Lexer):
    def lex(self, input):
        if not isinstance(input, InputStream): input = InputStream(str(input))
        while input.peek() in WHITESPACE:
            input.next()

class EndStatementLexer(Lexer):
    def lex(self, input):
        if not isinstance(input, InputStream): input = InputStream(str(input))
        while input.peek() in ENDSTATEMENT:
            input.next()
            
class IdentifierLexer(Lexer):
    class Exception(LexerException): pass
    class Identifier(object):
        def __init__(self, name):
            self.name = name
        def __repr__(self):
            return self.name
    def lex(self, input):
        if not isinstance(input, InputStream): input = InputStream(str(input))
        state = IdentifierLexer.Identifier('')
        if input.peek() in ALPHA:
            state.name = input.next()
            while input.peek() in ALPHANUMERIC:
                state.name += input.next()
        else: return None
        return state
        
class StringLexer(Lexer):
    class Exception(LexerException): pass
    class String(object):
        def __init__(self, text):
            self.text = text
        def __repr__(self):
            return self.text     
    def lex(self, input):
        if not isinstance(input, InputStream): input = InputStream(str(input))
        state = StringLexer.String('')
        if input.peek() == "'":
            input.next()
            while True:
                if input.peek() ==  InputStream.End: break
                if input.peek() == "'":
                    input.next()
                    break
                state.text += input.next()
            input.next()
        elif input.peek() == '"':
            input.next()
            while True:
                if input.peek() ==  InputStream.End: break
                if input.peek() == '"':
                    input.next()
                    break
                state.text += input.next()
            input.next()
        else: return None
        return state
    
class CommentLexer(Lexer):
    class Exception(LexerException): pass
    class Comment(object):
        def __init__(self, text):
            self.text = text
        def __repr__(self):
            return self.text
    def lex(self, input):
        if not isinstance(input, InputStream): input = InputStream(str(input))
        state = CommentLexer.Comment('')
        if input.peek(0,2) == '//':
            input.next(2)
            while True:
                if input.peek() ==  InputStream.End: break
                if input.peek() == '\n':
                    input.next()
                    break
                
                state.text += input.next()
        elif input.peek(0,2) == '/*':
            input.next(2)
            while True:
                if input.peek() ==  InputStream.End: break
                if input.peek(0,2) == '*/':
                    input.next(2)
                    break
                state.text += input.next()
        else: return None
        return state
                

class NumberLexer(Lexer):
    class Exception(LexerException): pass    
    class Number(object):
        def __init__(self, text, radix, length, exponent, type):
            self.text = text
            self.radix = radix
            self.length = length
            self.exponent = exponent
            self.type = type    
        def __repr__(self):
            return 'text %s radix %s length %s exponent %s type %s' % (self.text, self.radix, self.length, self.exponent, self.type)
    def lex(self, input):
        if not isinstance(input, InputStream): input = InputStream(str(input))
        state = NumberLexer.Number('', None, None, None, None)
        if not input.peek() in NUMERIC+'.'+'-': return None
        state.radix = 10
        if input.peek() == '0':
            if input.peek(1) in 'xX':
                state.radix = 16
                input.next(2)
            elif input.peek(1) in 'bB':
                state.radix = 2
                input.next(2)
        if state.radix == 10:
            if input.peek() == '-':
                state.text += input.next()
            while input.peek() in NUMERIC+'.'+'eE':
                if input.peek() == '.':
                    if '.' in state.text: raise NumberLexer.Exception()
                if input.peek() in 'eE':
                    if state.exponent != None: raise NumberLexer.Exception()
                    state.exponent = ''
                    input.next()
                    continue
                if state.exponent == None: state.text += input.next()
                else: state.exponent += input.next()
        elif state.radix == 16:
            while input.peek() in HEX:
                state.text += input.next()
        elif state.radix == 2:
            while input.peek() in '01':
                state.text += input.next()                
        if input.peek() == '#':
            input.next()
            if input.peek() not in 'usfUSF': raise NumberLexer.Exception()
            state.type = input.next().lower()
            if state.type == 'f' and not state.radix == 10: raise NumberLexer.Exception()
        if state.type:
            if input.peek(0, 1) == '8':
                input.next(1)
                state.length = 8
            if input.peek(0, 2) == '16':
                input.next(2)
                state.length = 16
            if input.peek(0, 2) == '32':
                input.next(2)
                state.length = 32
            if input.peek(0, 2) == '64':
                input.next(2)
                state.length = 64
        else:
            if state.radix == 10 and '.' in state.text: state.type = 'f'
            elif state.radix == 10 and state.exponent != None: state.type = 'f'
            else: state.type  = 's'
            state.length = 32
        state.text = state.text.lower()
        return state
    
number = NumberLexer()
comment = CommentLexer()
string = StringLexer()
identifier = IdentifierLexer()
operator = OperatorLexer()
whitespace = WhitespaceLexer()
endstatement = EndStatementLexer()

class Parser(object): pass

class AssemblerParser(Parser):
    class Exception(LexerException): pass
    def guess(self, input):
        if input.peek() in ENDSTATEMENT+InputStream.End: return endstatement
        if input.peek() == InputStream.End: return InputStream.End
        if input.peek() in WHITESPACE: return whitespace
        if input.peek(0,2) == '/*': return comment
        if input.peek(0,2) == '//': return comment    
        if input.peek() in OPERATORS: return operator
        if input.peek() in NUMERIC+'.'+'-': return number
        if input.peek() in '"\'': return string
        if input.peek() in ALPHA: return identifier
        raise AssemblerParser.Exception()
    def skip(self, input):
        lexer = self.guess(input)
        while lexer in (comment, whitespace): 
            token = lexer.lex(input)
            if input.peek() == InputStream.End: return False
            lexer = self.guess(input)
        return True
        
    def parse(self, input):
        if not isinstance(input, InputStream): input = InputStream(str(input))
        lexer = self.guess(input)
        while input.peek() != InputStream.End:
            self.skip(input)
            lexer = self.guess(input)
            if lexer == endstatement:
                lexer.lex(input)
                continue
            if not lexer == identifier: raise AssemblerParser.Exception('Expected %s but found %s' % (type(identifier), type(lexer)))
            instruction = lexer.lex(input)
            print 'instruction %s' % (instruction)
            self.skip(input)
            lexer = self.guess(input)
            while not lexer == endstatement:
                parameter = lexer.lex(input)
                print 'parameter %s %s %s' % (parameter, type(parameter), type(lexer))
                self.skip(input)
                lexer = self.guess(input)
                
assembler = AssemblerParser()

"""
Two states: source and destination.
Expression start in source.

Two actions: move and apply.

Moving:
src -> dest (dest then becomes result and src)

Applying:
src => dest (dest then becomes result and src)

Instances:
Everything in the system is an instance. For object instances, types, stack frames and operators are instances,
as well as other more conventional items such as numbers, strings, and instances.

Fundamental value types:
Fundamental types are numeric types, booleans, undefined and none types, and strings.

Fundamental container types:
Containers which can hold other instances are dictionaries, lists, object instances, and stack frames.

Stack frame:

The instructions operate in the context of a stack frame. The stack frame contains a dictionary for holding instances,
and a stack for holding temporary instances. It also contains a lookup list which affects how instances are looked up by name.

Manipulation of the current stack frame stack:

to push something to the stack:
x -> push stack top 

to pop something from the stack:
pop stack top -> x



Locations:      
    stack locations:
        dest:
            stack top, stack bottom, stack index negative, stack index positive (in set and push flavors)
        source:
            stack top, stack bottom, stack index negative, stack index positive  (in get and pop flavors)
    instance lookup:
        dest:
            local, global, runtime, variable 
        source:
            local, global, runtime, variable 
    builtins:
        source:
            immediate-[numeric type], immediate-true, immediate-false, immediate-none, immediate-undefined

Execution frame:
    pointer to program
    pointer to lookup stack
    pointer to variable stack
    
Per thread execution stack:
    Only holds execution frames.
    
    push src
        Pushes a execution frame to the stack. Next program cycle, it will be pointing to this new frame.
    pop src src
        Pops execution frames until the 'src' frame is encountered. The 'result'
        
Lookup stack:
    Holds dict like objects
    
    push 
    set src -> dest
    del dest
    apply source -> dest
    


"""
class Bitfield(object):
    def __init__(self, bits, names):
        self.bits = bits
        self.names = names
        
class Combinator(object):
    def __init__(self, combinators):
        self.combinators = conbinators

class Decider(object):
    def __init__(self, combinators):
        self.combinators = conbinators
""" apply instance default 'a' => operator lookup 'b'"""
     
class Instruction(object):
    action = Bitfield((0,),('move', 'apply'))
    target = Bitfield((1,2,3,),('local_stack', 'execution_stack', 'lookup_stack', 'instance', 'result', 'operator'))
    local_stack_read = Bitfield((4,5),'get','pop')
    local_stack_write = Bitfield((4,5),'set','push')
    local_stack_targets = Bitfield((6,7,8),('top', 'bottom', 'positive', 'negative'))
    lookup_stack_read = Bitfield((4,5),'get','pop')
    lookup_stack_write = Bitfield((4,5),'set','push')
    lookup_stack_actions = Bitfield((6,7,8),('top', 'bottom', 'positive', 'negative'))    
    execution_stack_read =  Bitfield((4,5,6),('top', 'bottom', 'positive', 'negative'))
    execution_stack_write = Bitfield((4,5),'push','return')
    execution_stack_targets =  Bitfield((6,7,8),('top', 'bottom', 'positive', 'negative'))
    instance_src = Bitfield((4,5), ('global', 'local', 'default'))
    instance_src = Bitfield((4,5), ('global', 'local', 'default'))
    
    local_stack_read = Combinator(local_stack_read, local_stack_targets)
    local_stack_write = Combinator(local_stack_write, local_stack_targets)
    lookup_stack_read = Combinator(lookup_stack_read, lookup_stack_targets)
    lookup_stack_write = Combinator(lookup_stack_write, lookup_stack_targets)
    read_targets = Decider(target, {'local_stack':local_stack_read, 'execution_stack':execution_stack_read})
    
    def __init__(self):
        pass
    
class Push(Instruction): 
class Pop(Instruction): pass
class Load(Instruction): pass
class Program(object):
    def __init__(self):
        self.strings = {}
        self.files = {}
        
    def string(self, str):
        if not str in self.strings:
            self.strings[str] = len(self.strings)
        return self.strings[str]
    def file(self, file):
        if not str in self.strings:
            self.strings[str] = len(self.strings)
        return self.strings[str]
            
def testOperatorLexer():
    test = '+-+/++>>>>>><<>><<<<<<'
    tests = (
        OperatorLexer.Operator('+'),
        OperatorLexer.Operator('-'),
        OperatorLexer.Operator('+'),
        OperatorLexer.Operator('/'),
        OperatorLexer.Operator('++'),
        OperatorLexer.Operator('>>>'),
        OperatorLexer.Operator('>>>'),
        OperatorLexer.Operator('<<'),
        OperatorLexer.Operator('>>'),
        OperatorLexer.Operator('<<<'),
        OperatorLexer.Operator('<<<')
    ) 
    i = InputStream(test)
    n = 0
    while True:
        op = operator.lex(i)
        if op == None: break
        expected = tests[n]
        assert expected.operator == op.operator
        n+=1
        
def testWhitespaceLexer():
    input = InputStream('  \t \r   a   \t \r')
    whitespace.lex(input)
    assert identifier.lex(input).name == 'a'
    whitespace.lex(input)
    assert input.peek() == InputStream.End
    
def testEndStatementLexer():
    input = InputStream(';;;a\n\na;')
    endstatement.lex(input)
    assert identifier.lex(input).name == 'a'
    endstatement.lex(input)
    assert identifier.lex(input).name == 'a'
    endstatement.lex(input)
    assert input.peek() == InputStream.End

def testAssemblerParser():
    test="""
          //comment
    /* annother comment */  +  5.462
      abcd ^  ee435 * "abc 123" /* comment */ ' abcd ' ++
      ;
"""
    input = InputStream(test)
    while input.peek() != InputStream.End:
        lexer = assembler.guess(input)
        lexer.lex(input)
        
    test="""
load 5
set 10
load a 10 5;
set 6 c;
/* comment */


load 10 4
"""
    assembler.parse(test)
def testIdentifierLexer():
    assert identifier.lex('ab1c2de').name == 'ab1c2de'
    
def testStringLexer():
    assert string.lex("'yo dude'").text == 'yo dude'
    assert string.lex('"yo dude"').text == 'yo dude'
    assert string.lex("'yo dude").text == 'yo dude'
    
def testCommentLexer():
    assert comment.lex('//abcd1').text == 'abcd1'
    assert comment.lex('//abcd2\nsdddd').text == 'abcd2'
    assert comment.lex('/*abcd3*/').text == 'abcd3'
    assert comment.lex('/*abcd3\nabcd4*/').text == 'abcd3\nabcd4'
    assert comment.lex('/*abcd5').text == 'abcd5'

def testNumberLexer():
    tests = (
         ('123',                NumberLexer.Number('123', 10, 32, None, 's')),
         ('123#s32',            NumberLexer.Number('123', 10, 32, None, 's')),
         ('123#s64',            NumberLexer.Number('123', 10, 64, None, 's')),
         ('123#u64',            NumberLexer.Number('123', 10, 64, None, 'u')),
         ('123#u32',            NumberLexer.Number('123', 10, 32, None, 'u')),
         ('123#u8',             NumberLexer.Number('123', 10, 8, None, 'u')),
         ('123#U8',             NumberLexer.Number('123', 10, 8, None, 'u')),
         ('0x123abc#s32',       NumberLexer.Number('123abc', 16, 32, None, 's')),
         ('0x123abc#s64',       NumberLexer.Number('123abc', 16, 64, None, 's')),
         ('0X123ABC#S64',       NumberLexer.Number('123abc', 16, 64, None, 's')),
         ('0x123abc#u64',       NumberLexer.Number('123abc', 16, 64, None, 'u')),
         ('0x123abc#u32',       NumberLexer.Number('123abc', 16, 32, None, 'u')),
         ('0b10111#u8',         NumberLexer.Number('10111', 2, 8, None, 'u')),
         ('0b10111#s32',        NumberLexer.Number('10111', 2, 32, None, 's')),
         ('0b10111#s64',        NumberLexer.Number('10111', 2, 64, None, 's')),
         ('0b10111#u64',        NumberLexer.Number('10111', 2, 64, None, 'u')),
         ('0b10111#u32',        NumberLexer.Number('10111', 2, 32, None, 'u')),
         ('0B10111#U32',        NumberLexer.Number('10111', 2, 32, None, 'u')),
         ('0b10111#u8',         NumberLexer.Number('10111', 2, 8, None, 'u')),
         ('1.23',               NumberLexer.Number('1.23', 10, 32, None, 'f')),
         ('.123',               NumberLexer.Number('.123', 10, 32, None, 'f')), 
         ('-.123',              NumberLexer.Number('-.123', 10, 32, None, 'f')),
         ('123e10',             NumberLexer.Number('123', 10, 32, '10', 'f')),
         ('.123e10',            NumberLexer.Number('.123', 10, 32, '10', 'f')),
         ('-.123e10',           NumberLexer.Number('-.123', 10, 32, '10', 'f')),
         ('.123#f64',           NumberLexer.Number('.123', 10, 64, None, 'f')),
         ('.123#F64',           NumberLexer.Number('.123', 10, 64, None, 'f')),  
         ('-.123#f64',          NumberLexer.Number('-.123', 10, 64, None, 'f')),
         ('123e10#f64',         NumberLexer.Number('123', 10, 64, '10', 'f')),
         ('.123e10#f64',        NumberLexer.Number('.123', 10, 64, '10', 'f')),
         ('.123E10#f64',        NumberLexer.Number('.123', 10, 64, '10', 'f')),
         ('-.123e10#f64',       NumberLexer.Number('-.123', 10, 64, '10', 'f'))
    )
    for text,expected in tests: 
        lexed = number.lex(text)
        #print lexed, expected
        assert lexed.text == expected.text
        assert lexed.radix == expected.radix
        assert lexed.length == expected.length
        assert lexed.exponent == expected.exponent
        assert lexed.type == expected.type   
        
def testInputStream():
    i = InputStream('abcdef')
    assert i.peek(0) == 'a'
    assert i.peek(1) == 'b'
    assert i.peek(0,3) == 'abc'
    assert i.next(4) == 'abcd'
    assert i.peek(0) == 'e'
    assert i.peek(1) == 'f'
    assert i.next(1) == 'e'
    assert i.peek(0,1) == 'f'
    assert i.peek(0,2) == 'f' + InputStream.End
    assert i.peek(1) == '' + InputStream.End
    assert i.next(2) == 'f' + InputStream.End
    assert i.next(1) == '' + InputStream.End
    assert i.peek(0) == '' + InputStream.End
        
testInputStream()
testNumberLexer()
testCommentLexer()
testStringLexer()
testIdentifierLexer()
testOperatorLexer()
testWhitespaceLexer()
testEndStatementLexer()
testAssemblerParser()